#! /usr/bin/python
#
# Stephen Poletto (spoletto)
# Peter Wilmot (pbwilmot)
# CSCI1580 - Web Search
# Spring 2011 - Brown University
#
# There are a lot of questions on this
# assignment, asking us to gather stats
# on how well our classifier performs
# given different feature sets, what
# percentage we mis-classify, etc. Rather
# than doing it all by hand, this script
# will allow us to automate the process.

from documentVector import *
import random
import numpy
import sys
import os

asgnDir = "/course/cs158/data/classif/"
stopWords = asgnDir + "stopWords.dat"
features1 = asgnDir + "features1.dat"
features2 = asgnDir + "features2.dat"
fullCollection = asgnDir + "fullCollection.dat"
test = asgnDir + "test.dat"
training1 = asgnDir + "training1.dat"
training2 = asgnDir + "training2.dat"
vecrep = "./vecrep.sh"
classify = "./classify.sh"
vecOutput = "vecoutput"
classifierOutput = "classifierOutput"

which = raw_input("Enter 1 to test the classifier, 2 to profile the vector output.\n")
if (which != '1' and which != '2'):
    print "You did not select an available option. Exiting."
    sys.exit()
    
if (which == '1'):

    classifierAlgo = raw_input("Would you like to test mnb or r?\n")
    if (classifierAlgo != "mnb" and classifierAlgo != "r"):
        print "Specified classifier algorithm does not exist. Exiting."
        sys.exit()
    
    featureID = raw_input("What feature set would you like to use: 1 or 2?\n")
    if (featureID != '1' and featureID != '2'):
        print "Specified feature set does not exist. Exiting."
        sys.exit()
    if (featureID == '1'):
        featureToUse = features1
    else:
        featureToUse = features2
    
    trainingID = raw_input("What training set would you like to use: 1 or 2?\n")
    if (trainingID != '1' and trainingID != '2'):
        print "Specified training set does not exist. Exiting."
        sys.exit()
    if (trainingID == '1'):
        trainingToUse = training1
    else:
        trainingToUse = training2
        
    vecInput = raw_input("What's the filename to the vector rep you'd like to use?\n")
    docsToClassify = raw_input("Do you want to check against (1) training1, (2) training2 or (3) test?\n")
    if docsToClassify == '1':
        docsToClassify = training1
    elif docsToClassify == '2':
        docsToClassify = training2
    elif docsToClassify == '3':
        docsToClassify = test
    else:
        print "Can't test against that input. Exiting."
        sys.exit()
        
    # Docs to classify won't quite work out just being passed in to the classify script.
    # We need to script off the pre-classifications from each line.
    newInputFilename = 'tmp' + str(int(random.random() * 500))
    newInputToClassify = open(newInputFilename, 'wb')
    oldInputToClassify = open(docsToClassify, 'r')
    for line in oldInputToClassify:
        docID = line.split(' ')[0]
        newInputToClassify.write(str(docID) + "\n")
    oldInputToClassify.close()
    newInputToClassify.close()
    
    outputFilename = classifierOutput + featureID + trainingID + ".dat"
        
    cmd = classify + " -" + classifierAlgo + " " + featureToUse + " " + vecInput + " " + trainingToUse + " " + newInputFilename + " " + outputFilename
    print "time " + cmd
    os.system("time " + cmd)
    
    # Alright, what was our percentage?
    
    # Number of docs to be classified:
    fout = os.popen('cat ' + outputFilename + ' | wc -l')
    numDocsClassified = int(fout.read().rstrip('\n'))
    
    # Number of docs differing from provided classifications
    fout = os.popen('sdiff -B -b -s ' + outputFilename + " " + docsToClassify + " | wc -l")
    numDocsDiffer = int(fout.read().rstrip('\n'))
    
    print "PERCENTAGE MATCHED = " + str((1.0 - (numDocsDiffer + 0.0)/(numDocsClassified + 0.0)) * 100)
    
else:
    
    rebuild = raw_input("Would you like to rebuild the vecrep output? (Y or N)\n")
    if rebuild == 'y' or rebuild == 'Y':
        featureID = raw_input("What feature set would you like to use: 1 or 2?\n")
        if (featureID != '1' and featureID != '2'):
            print "Specified feature set does not exist. Exiting."
            sys.exit()
        if (featureID == '1'):
            featureToUse = features1
        else:
            featureToUse = features2
        filename = vecOutput + featureID + ".dat"
        cmd = vecrep + " " + stopWords + " " + fullCollection + " " + featureToUse + " " + filename
        print "time " + cmd
        os.system("time " + cmd) 
    else:
        filename = raw_input("Since we're not rebuilding the output, what file should we use?\n")
        featureID = raw_input("And which feature was this for? (1 or 2?)\n")
        if (featureID != '1' and featureID != '2'):
            print "Specified feature set does not exist. Exiting."
            sys.exit()
        if (featureID == '1'):
            featureToUse = features1
        else:
            featureToUse = features2
        
    # The file should exist by now, no matter which path we took.
    vectorFile = open(filename, 'r')
    featuresFile = open(featureToUse, 'r')
    
    featureCount = 0
    for line in featuresFile:
        featureCount += 1
    featuresFile.close()
    
    docIDToVector = {}
    for vecrep in vectorFile.readlines():
        vecrep = vecrep.rstrip("\n")
        vector = DocumentVector(vecrep[:-1], featureCount)
        docIDToVector[vector.docID] = vector
    vectorFile.close()
    
    # First, the question asks us to find the average number of features per document.
    totalFeaturesSeen = 0
    docCount = 0
    for doc in docIDToVector:
        totalFeaturesSeen += numpy.sum(docIDToVector[doc].countVector())
        docCount += 1
    print "AVERAGE NUMBER OF FEATURES PER DOCUMENT : " + str((totalFeaturesSeen + 0.0)/docCount)
    
    # Then, we need the number of documents containing each feature. We might
    # even want to build a graph here.
    for featureID in range(0, featureCount):
        numberOfDocsAppearedIn = 0
        for doc in docIDToVector:
            if docIDToVector[doc].countVector()[featureID] > 0:
                numberOfDocsAppearedIn += 1
        print "FEATURE ID " + str(featureID) + " OCCURRED IN " + str(numberOfDocsAppearedIn) + " DOCS."
        
        